{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 初始化\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import sys\n",
    "import os\n",
    "os.chdir('E:\\GitHub\\QA-abstract-and-reasoning')\n",
    "sys.path.append('E:\\GitHub\\QA-abstract-and-reasoning')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from utils.config import TRAIN_DATA\n",
    "pd.set_option(\"display.max_rows\",200)\n",
    "pd.set_option(\"display.max_colwidth\",500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def detect_bad_words(x):\n",
    "    for bad in bad_words:\n",
    "        if (bad in x and len(x)<6):\n",
    "            return True\n",
    "    return False\n",
    "train = pd.read_csv(TRAIN_DATA).fillna(\"\")\n",
    "train[\"QD_nstr\"] = train[\"Question\"].apply(lambda x: len(x)) + train[\"Dialogue\"].apply(lambda x: len(x))\n",
    "train[\"Rp_nstr\"] = train[\"Report\"].apply(lambda x: len(x))\n",
    "bad_words = ['参照下图', '参照图片', '参照图文', '参照图文',\n",
    "             '详见图片', '长时间不回复', '如图', '按图',\n",
    "             '看图', '见图', '随时联系', '已解决', '已经提供图片',\n",
    "             '已经发图片', '还在吗', '匹配']\n",
    "train[\"bad_words\"] = train[\"Report\"].apply(lambda x: detect_bad_words(x))\n",
    "\n",
    "train[\"qiche_master\"] = train[\"Report\"].apply(lambda x: \"建议您下载汽车大师APP\" in x)\n",
    "\n",
    "bad_sample_index = train[((train[\"QD_nstr\"] >= 400) &  # Quesetion Dialogue 字符数>=400，且\n",
    "                          (train[\"Rp_nstr\"] <= 8)) |  # Report字符数<=8(882)，或\n",
    "                         train[\"bad_words\"] |  # 回复包括bad词(643)，或\n",
    "                         (train[\"Rp_nstr\"] < 2) | # Report字符数<2(84)，或\n",
    "                        train[\"qiche_master\"]  # 回复推销汽车大师app(31)\n",
    "                        ].index  # 共1482"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(882, 10)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[(train[\"QD_nstr\"] >= 400) & (train[\"Rp_nstr\"] <= 8)].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1482"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(bad_sample_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tf2.0]",
   "language": "python",
   "name": "conda-env-tf2.0-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
